
# Import required libraries and dependencies
import pandas as pd
import hvplot.pandas
from pathlib import Path
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler


# Load the data into a Pandas DataFrame
df_market_data = pd.read_csv(
    Path("Resources/crypto_market_data.csv"),
    index_col="coin_id")

# Display sample data
df_market_data.head(10)


# Generate summary statistics
df_market_data.describe()


# Plot your data to see what's in your DataFrame
df_market_data.hvplot.line(
    width=800,
    height=400,
    rot=90
)


# Use the `StandardScaler()` module from scikit-learn to normalize the data from the CSV file
scaled_data = StandardScaler().fit_transform(df_market_data)


# Create a DataFrame with the scaled data
df_market_data_scaled = pd.DataFrame(
    scaled_data,
    columns=df_market_data.columns
)

# Copy the crypto names from the original data
df_market_data_scaled["coin_id"] = df_market_data.index

# Set the coinid column as index
df_market_data_scaled = df_market_data_scaled.set_index("coin_id")

# Display sample data
df_market_data_scaled.head()


# Create a list with the number of k-values to try
# Use a range from 1 to 11

k = list(range(1,11))


# Create an empy list to store the inertia values
inertia = []


# Create a for loop to compute the inertia with each possible value of k
# Inside the loop:
# 1. Create a KMeans model using the loop counter for the n_clusters
# 2. Fit the model to the data using `df_market_data_scaled`
# 3. Append the model.inertia_ to the inertia list

for i in k:
    model = KMeans(n_clusters=i, random_state=0)
    model.fit(df_market_data_scaled)
    inertia.append(model.inertia_)

C:\Users\keito\anaconda3\envs\dev\lib\site-packages\sklearn\cluster\_kmeans.py:1037: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.
  "KMeans is known to have a memory leak on Windows "


# Create a dictionary with the data to plot the Elbow curve
elbow_data = {'k':k,'inertia':inertia}

# Create a DataFrame with the data to plot the Elbow curve
df_elbow = pd.DataFrame(elbow_data)


# Plot a line chart with all the inertia values computed with 
# the different values of k to visually identify the optimal value for k.
elbow_data_line = df_elbow.hvplot.line(x='k',y='inertia',title='Elbow Curve using KMeans', label='using KMeans',height=500, width=800)
elbow_data_line


# Initialize the K-Means model using the best value for k
model = KMeans(n_clusters=4)


# Fit the K-Means model using the scaled data
model.fit(df_market_data_scaled)

KMeans(n_clusters=4)


# Predict the clusters to group the cryptocurrencies using the scaled data
crypt_clusters = model.predict(df_market_data_scaled)

# View the resulting array of cluster values.
crypt_clusters

array([0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0,
       1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 3, 0, 1, 1, 2, 1, 1, 1, 1])


# Create a copy of the DataFrame
df_market_data_scaled_predictions = df_market_data_scaled.copy()


# Add a new column to the DataFrame with the predicted clusters
df_market_data_scaled_predictions['crypt_clusters'] = crypt_clusters

# Display sample data
df_market_data_scaled_predictions.head(20)


# Create a scatter plot using hvPlot by setting 
# `x="price_change_percentage_24h"` and `y="price_change_percentage_7d"`. 
# Color the graph points with the labels found using K-Means and 
# add the crypto name in the `hover_cols` parameter to identify 
# the cryptocurrency represented by each data point.

df_market_data_scaled_predictions_plot = df_market_data_scaled_predictions.hvplot.scatter(x='price_change_percentage_24h',y='price_change_percentage_7d',by='crypt_clusters',hover_cols=['coin_id'], title='Scatter Plot by Crypt Segment - k=4', height=500, width=800)
df_market_data_scaled_predictions_plot


# Create a PCA model instance and set `n_components=3`.
pca = PCA(n_components=3)


# Use the PCA model with `fit_transform` to reduce to 
# three principal components.
crypt_pca_data = pca.fit_transform(df_market_data_scaled)

# View the first five rows of the DataFrame. 
crypt_pca_data[:5]

array([[-0.60066733,  0.84276006,  0.46159457],
       [-0.45826071,  0.45846566,  0.95287678],
       [-0.43306981, -0.16812638, -0.64175193],
       [-0.47183495, -0.22266008, -0.47905316],
       [-1.15779997,  2.04120919,  1.85971527]])


# Retrieve the explained variance to determine how much information 
# can be attributed to each principal component.
pca.explained_variance_ratio_

array([0.3719856 , 0.34700813, 0.17603793])


# Create a new DataFrame with the PCA data.
# Note: The code for this step is provided for you

# Creating a DataFrame with the PCA data
df_crypto_pca = pd.DataFrame(crypt_pca_data, columns=['PC1','PC2','PC3'])

# Copy the crypto names from the original data
df_crypto_pca['coin_id'] = df_market_data.index

# Set the coinid column as index
df_crypto_pca = df_crypto_pca.set_index('coin_id')

# Display sample data
df_crypto_pca.head(10)


# Create a list with the number of k-values to try
# Use a range from 1 to 11
k = list(range(1,11))


# Create an empy list to store the inertia values
inertia = []


# Create a for loop to compute the inertia with each possible value of k
# Inside the loop:
# 1. Create a KMeans model using the loop counter for the n_clusters
# 2. Fit the model to the data using `df_market_data_pca`
# 3. Append the model.inertia_ to the inertia list
for i in k:
    model = KMeans(n_clusters=i, random_state=0)
    model.fit(df_crypto_pca)
    inertia.append(model.inertia_)

C:\Users\keito\anaconda3\envs\dev\lib\site-packages\sklearn\cluster\_kmeans.py:1037: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.
  "KMeans is known to have a memory leak on Windows "


# Create a dictionary with the data to plot the Elbow curve
elbow_data_pca = {'k':k,'inertia':inertia}

# Create a DataFrame with the data to plot the Elbow curve
df_elbow_pca = pd.DataFrame(elbow_data_pca)


# Plot a line chart with all the inertia values computed with 
# the different values of k to visually identify the optimal value for k.
elbow_plot_pca = df_elbow_pca.hvplot.line(x='k',y='inertia',title='Elbow Curve Using PCA Data', label='using PCA',color='red',height=500, width=800)
elbow_plot_pca


# Compare original elbow curve with it using PCA
combined_plot = elbow_data_line * elbow_plot_pca
combined_plot.opts(legend_position='right')


# Initialize the K-Means model using the best value for k
model = KMeans(n_clusters=4, random_state=0)


# Fit the K-Means model using the PCA data
model.fit(df_crypto_pca)

KMeans(n_clusters=4, random_state=0)


# Predict the clusters to group the cryptocurrencies using the PCA data
k_4 = model.predict(df_crypto_pca)

# View the resulting array of cluster values.
k_4

array([0, 0, 3, 3, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0, 3, 0, 3, 3, 0, 3, 3, 0,
       3, 3, 3, 3, 3, 3, 0, 3, 3, 3, 1, 0, 3, 3, 2, 3, 3, 3, 3])


# Create a copy of the DataFrame with the PCA data
crypto_predictions_pca_df = df_crypto_pca.copy()

# Add a new column to the DataFrame with the predicted clusters
crypto_predictions_pca_df['crypto_segments'] = k_4

# Display sample data
crypto_predictions_pca_df.head(10)


# Create a scatter plot using hvPlot by setting 
# `x="PC1"` and `y="PC2"`. 
# Color the graph points with the labels found using K-Means and 
# add the crypto name in the `hover_cols` parameter to identify 
# the cryptocurrency represented by each data point.
crypto_predictions_pca_plot = crypto_predictions_pca_df.hvplot.scatter(x='PC1',y='PC2',by='crypto_segments',hover_cols='coin_id',title='Plot PCA using KMeans',height=500, width=800)
crypto_predictions_pca_plot


# Composite plot to contrast the Elbow curves
elbow_data_line + elbow_plot_pca


# Compoosite plot to contrast the clusters
df_market_data_scaled_predictions_plot + crypto_predictions_pca_plot

	price_change_percentage_24h	price_change_percentage_7d	price_change_percentage_14d	price_change_percentage_30d	price_change_percentage_60d	price_change_percentage_200d	price_change_percentage_1y
coin_id
bitcoin	1.08388	7.60278	6.57509	7.67258	-3.25185	83.51840	37.51761
ethereum	0.22392	10.38134	4.80849	0.13169	-12.88890	186.77418	101.96023
tether	-0.21173	0.04935	0.00640	-0.04237	0.28037	-0.00542	0.01954
ripple	-0.37819	-0.60926	2.24984	0.23455	-17.55245	39.53888	-16.60193
bitcoin-cash	2.90585	17.09717	14.75334	15.74903	-13.71793	21.66042	14.49384
binancecoin	2.10423	12.85511	6.80688	0.05865	36.33486	155.61937	69.69195
chainlink	-0.23935	20.69459	9.30098	-11.21747	-43.69522	403.22917	325.13186
cardano	0.00322	13.99302	5.55476	10.10553	-22.84776	264.51418	156.09756
litecoin	-0.06341	6.60221	7.28931	1.21662	-17.23960	27.49919	-12.66408
bitcoin-cash-sv	0.92530	3.29641	-1.86656	2.88926	-24.87434	7.42562	93.73082

	price_change_percentage_24h	price_change_percentage_7d	price_change_percentage_14d	price_change_percentage_30d	price_change_percentage_60d	price_change_percentage_200d	price_change_percentage_1y
count	41.000000	41.000000	41.000000	41.000000	41.000000	41.000000	41.000000
mean	-0.269686	4.497147	0.185787	1.545693	-0.094119	236.537432	347.667956
std	2.694793	6.375218	8.376939	26.344218	47.365803	435.225304	1247.842884
min	-13.527860	-6.094560	-18.158900	-34.705480	-44.822480	-0.392100	-17.567530
25%	-0.608970	0.047260	-5.026620	-10.438470	-25.907990	21.660420	0.406170
50%	-0.063410	3.296410	0.109740	-0.042370	-7.544550	83.905200	69.691950
75%	0.612090	7.602780	5.510740	4.578130	0.657260	216.177610	168.372510
max	4.840330	20.694590	24.239190	140.795700	223.064370	2227.927820	7852.089700

	price_change_percentage_24h	price_change_percentage_7d	price_change_percentage_14d	price_change_percentage_30d	price_change_percentage_60d	price_change_percentage_200d	price_change_percentage_1y
coin_id
bitcoin	0.508529	0.493193	0.772200	0.235460	-0.067495	-0.355953	-0.251637
ethereum	0.185446	0.934445	0.558692	-0.054341	-0.273483	-0.115759	-0.199352
tether	0.021774	-0.706337	-0.021680	-0.061030	0.008005	-0.550247	-0.282061
ripple	-0.040764	-0.810928	0.249458	-0.050388	-0.373164	-0.458259	-0.295546
bitcoin-cash	1.193036	2.000959	1.760610	0.545842	-0.291203	-0.499848	-0.270317

	price_change_percentage_24h	price_change_percentage_7d	price_change_percentage_14d	price_change_percentage_30d	price_change_percentage_60d	price_change_percentage_200d	price_change_percentage_1y	crypt_clusters
coin_id
bitcoin	0.508529	0.493193	0.772200	0.235460	-0.067495	-0.355953	-0.251637	0
ethereum	0.185446	0.934445	0.558692	-0.054341	-0.273483	-0.115759	-0.199352	0
tether	0.021774	-0.706337	-0.021680	-0.061030	0.008005	-0.550247	-0.282061	1
ripple	-0.040764	-0.810928	0.249458	-0.050388	-0.373164	-0.458259	-0.295546	1
bitcoin-cash	1.193036	2.000959	1.760610	0.545842	-0.291203	-0.499848	-0.270317	0
binancecoin	0.891871	1.327295	0.800214	-0.057148	0.778653	-0.188232	-0.225533	0
chainlink	0.011397	2.572251	1.101647	-0.490495	-0.931954	0.387759	-0.018284	0
cardano	0.102530	1.508001	0.648885	0.328959	-0.486349	0.065080	-0.155428	0
litecoin	0.077497	0.334297	0.858520	-0.012646	-0.366477	-0.486266	-0.292351	0
bitcoin-cash-sv	0.448952	-0.190684	-0.248043	0.051634	-0.529666	-0.532961	-0.206029	1
crypto-com-chain	0.331280	-1.614844	-1.054521	-0.729931	-0.350155	-0.022866	-0.034570	1
usd-coin	0.034352	-0.733026	-0.023140	-0.065775	0.002925	-0.550599	-0.282232	1
eos	0.155710	-0.922491	0.115024	-0.237488	-0.642837	-0.508220	-0.296330	1
monero	0.262723	1.792602	2.202665	1.437842	0.893865	-0.155893	-0.167644	0
tron	0.130050	-0.041018	0.147155	-0.543776	0.120116	-0.241118	-0.234014	1
tezos	-0.151583	0.708196	0.258012	-0.602296	-0.956049	-0.449211	-0.168479	0
okb	-0.923203	-1.437359	-0.629963	-0.460558	-0.058504	-0.457283	-0.166900	1
stellar	-0.277543	-0.385209	-0.153243	-0.371816	-0.656403	-0.353387	-0.270874	1
cosmos	-0.255978	1.840274	0.643565	0.116538	-0.151913	-0.117565	-0.215191	0
cdai	0.180851	-0.704931	-0.001816	-0.143237	0.016060	-0.551146	-0.282310	1

	PC1	PC2	PC3
coin_id
bitcoin	-0.600667	0.842760	0.461595
ethereum	-0.458261	0.458466	0.952877
tether	-0.433070	-0.168126	-0.641752
ripple	-0.471835	-0.222660	-0.479053
bitcoin-cash	-1.157800	2.041209	1.859715
binancecoin	-0.516534	1.388377	0.804071
chainlink	-0.450711	0.517699	2.846143
cardano	-0.345600	0.729439	1.478013
litecoin	-0.649468	0.432165	0.600303
bitcoin-cash-sv	-0.759014	-0.201200	-0.217653

Module 10 Application¶

Challenge: Crypto Clustering¶

Import the Data¶

Prepare the Data¶

Find the Best Value for k Using the Original Data¶

Answer the following question: What is the best value for k?¶

Cluster Cryptocurrencies with K-means Using the Original Data¶

Optimize Clusters with Principal Component Analysis¶

Answer the following question: What is the total explained variance of the three principal components?¶

Find the Best Value for k Using the PCA Data¶

Answer the following questions: What is the best value for k when using the PCA data? Does it differ from the best k value found using the original data?¶

Cluster Cryptocurrencies with K-means Using the PCA Data¶

Visualize and Compare the Results¶

Answer the following question: After visually analyzing the cluster analysis results, what is the impact of using fewer features to cluster the data using K-Means?¶